package edu.unc.ils.memai.extract;

import java.io.File;


import java.io.FileWriter;
import java.io.Reader;
import java.io.StringReader;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.collections15.Bag;
import org.apache.commons.collections15.bag.HashBag;
import org.apache.commons.collections15.bag.TreeBag;
import org.apache.commons.io.FileUtils;
import org.apache.commons.math.stat.Frequency;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.LowerCaseFilter;
import org.apache.lucene.analysis.PorterStemFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.QueryParser.Operator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.openrdf.concepts.skos.core.Concept;
import org.openrdf.elmo.ElmoModule;
import org.openrdf.elmo.sesame.SesameManager;
import org.openrdf.elmo.sesame.SesameManagerFactory;
import org.openrdf.repository.Repository;
import org.openrdf.repository.sail.SailRepository;
import org.openrdf.sail.nativerdf.NativeStore;

// The index must match the query model, particularly stemming and stopping
public class LuceneCandidateExtractor 
{
    Directory dir = null;
    Map<String, String> idPrefLabelMap = new HashMap<String, String>();

    List<String> stopwords = new ArrayList<String>();
    
    class MyAnalyzer extends Analyzer {
        public final TokenStream tokenStream(String fieldName, Reader reader) {
            return new PorterStemFilter(
              new LowerCaseFilter(
                new StandardFilter(
                  new StandardTokenizer(Version.LUCENE_34, reader))));
        }
    }
      
    public static void main(String[] args) throws Exception
    {
        LuceneCandidateExtractor v = new LuceneCandidateExtractor();
        //v.dumpTermFrequencies("/Users/cwillis/dev/hive/hive-data/nalt/naltStore");
        v.loadStopwords("src/main/resources/stopwords_en.txt");
        v.importFromSesame("/Users/cwillis/dev/hive/hive-data/nalt/naltStore");
        
        
        String text = "Background: Cotton (Gossypium hirsutum L.) fibers are trichomes that initiate from the ovule epidermis. " + 
        "Little is known about the developmental pathway causing fiber to differentiate from ovular epidermal cells even " + 
        "though limits on the number of cells that differentiate into fiber will limit yield. Results: A method was developed " + 
        "to isolate RNA from fiber initials 1 day post anthesis (dpa). Complementary DNA libraries representing 1 dpa fibers " + 
        "and other cotton tissues were sequenced and analyzed. Assembly of G. hirsutum Expressed Sequenced Tags (ESTs) identified " + 
        "over 11,000 sequences not previously represented in GenBank. New genes identified among these ESTs were represented " + 
        "on microarrays. The microarrays were used to identify genes enriched in fiber initials (1 dpa fibers) and elongating " + 
        "fibers. Analyses of Gene Ontologies (GO) of differentially expressed genes determined that terms associated with the " + 
        "membranes were statistically over represented among genes increased in expression in fiber initials and 10 dpa fibers. " + 
        "Staining ovules with a fluorescent dye confirmed an increase in Endoplasmic Reticulum (ER) occurred in fiber initials on " + 
        "the day of anthesis, persisted through 3 dpa and was absent in a fiberless mutant. Two genes similar to the " + 
        "CAPRICE/TRIPTYCHON (CPC) gene that inhibits differentiation of leaf trichomes in Arabidopsis were also characterized. " + 
        "Genes associated with novel regulation of brassinosterols, GTP mediated signal transduction and cell cycle control and " +
        "components of a Ca+2 mediated signaling pathway were identified. Staining of cellular Ca+2 indicated that fiber initials " +
        "had more Ca+2 than other ovule cells supporting a role for Ca+2 in fiber development. Conclusion: Analysis of genes " + 
        "expressed in fiber initials identified a unique stage in fiber development characterized by an increase in ER and Ca+2 " + 
        "levels that occurred between 0 and 1 dpa. The gene similar to CPC has a MYB domain but appears to lack a transcription " + 
        "activating domain similar to the Arabisopsis gene. The method used to stain the ER also can be used to count fiber " + 
        "initials and showed fiber cells develop from adjacent cells unlike leaf trichomes.";
        v.test(text);
    }
    
    public void test(String text) throws Exception
    {
        
        String[] segments = text.split("\\. ");
        int n = 5;
        Bag<String> matched = new HashBag<String>();
        for (String segment: segments)
        {
            segment = segment.replaceAll(",", "");
            segment = segment.replaceAll(":", "");
            segment = segment.replaceAll("\\(", "");
            segment = segment.replaceAll("\\)", "");
            segment = segment.toLowerCase();

            String[] words = segment.split(" ");
            for (int i=0; i<words.length; i++)
            {
                String key = words[i];
                if (!stopwords.contains(words[i]))
                {
                    List<String> ids = lookup(words, key, i, i+n);
                    for (String id: ids)
                    {
                        matched.add(id);
                    }
                }
            }
        }
        for (String id: matched.uniqueSet())
        {
            String prefLabel = idPrefLabelMap.get(id);
            System.out.println(prefLabel + "|" + matched.getCount(id));
        }
    }
    
    public List<String> lookup(String[] words, String prefix, int start, int length) throws Exception
    {
        List<String> found = new ArrayList<String>();
        
        Bag<String> ids = getCandidates(prefix);
        if (ids != null)
        {
            for (String id: ids)
                found.add(id);
        }
       
        for (int i=start+1; i<length; i++)
        {
            if (i > words.length-1)
                continue;
            
            if (!stopwords.contains(words[i]))
            {
                String key = prefix + " " + words[i];            
                List<String> f = lookup(words,key, i,length);
                found.addAll(f);
            }
        }
        return found;
    }


    public void importFromSesame(String path) throws Exception
    {
        dir = new RAMDirectory();
        
        // StandardAnalyzer = StandardFilter (normalizes tokens extracted with StandardTokenizer,
        // LowerCaseFilter, StopFilter
        // StandardTokenizer:
        //    Splits words at punctuation characters, removing punctuation. 
        //       However, a dot that's not followed by whitespace is considered part of a token.
        //    Splits words at hyphens, unless there's a number in the token, 
        //       in which case the whole token is interpreted as a product number and is not split.
        //    Recognizes email addresses and internet hostnames as one token.     
        IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_34, 
                new MyAnalyzer());
        
        IndexWriter index = new IndexWriter(dir, iwc);        
        
        // Open the Sesame repository
        String indexes = "spoc,ospc";
        NativeStore store = new NativeStore(new File(path), indexes);
        Repository repository = new SailRepository(store);
        repository.initialize();
        ElmoModule module = new ElmoModule();
        SesameManagerFactory factory = new SesameManagerFactory(module, repository);
        SesameManager manager = factory.createElmoManager(); 
        

        for (Concept concept:  manager.findAll(Concept.class))
        {
            javax.xml.namespace.QName qname = concept.getQName();
            String uri = qname.getNamespaceURI() + qname.getLocalPart();

            Document document = new Document();
            Field cid = new Field("conceptId", uri, Field.Store.YES, Field.Index.NOT_ANALYZED);
            document.add(cid);
            
            // Pref label
            String prefLabel = concept.getSkosPrefLabel();
            Field fpl = new Field("prefLabel", prefLabel, Field.Store.YES, 
                    Field.Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS);
            fpl.setBoost(2.0F);
            document.add(fpl);
            
            idPrefLabelMap.put(uri, prefLabel);
            
            
            // Alt labels
            Set<String> altLabels = concept.getSkosAltLabels();
            for (String altLabel: altLabels)
            {
                Field fal = new Field("altLabel", altLabel, Field.Store.YES, Field.Index.ANALYZED, 
                        TermVector.WITH_POSITIONS_OFFSETS);
                document.add(fal);
            }
            
            // Scope notes
            Set<Object> scopeNotes = concept.getSkosScopeNotes();
            for (Object scopeNote: scopeNotes)
            {
                String sn = (String)scopeNote;
                Field fsn = new Field("scopeNote", sn, Field.Store.NO, Field.Index.ANALYZED, 
                        TermVector.WITH_POSITIONS_OFFSETS);
                document.add(fsn);
            }

            index.addDocument(document);
        }
        repository.shutDown();
        store.shutDown();
        index.commit();
        index.close();
    }
    
    Map<String, Integer> termFrequency = new HashMap<String, Integer>();
    Map<String, Integer> documentFrequency = new HashMap<String, Integer>();
    
    public void countFrequencies(int docId, String phrase) throws Exception
    {
        StandardTokenizer tokenizer = new StandardTokenizer(Version.LUCENE_34, new StringReader(phrase));
        OffsetAttribute offsetAttribute = tokenizer.getAttribute(OffsetAttribute.class);
        CharTermAttribute charTermAttribute = tokenizer.getAttribute(CharTermAttribute.class);
        
        while (tokenizer.incrementToken())
        {
            int startOffset = offsetAttribute.startOffset();
            int endOffset = offsetAttribute.endOffset();
            String term = charTermAttribute.toString();
            
            Integer c1 = termFrequency.get(term);
            if (c1 == null)
                c1 = 0;
            c1++;
            termFrequency.put(term, c1);
            
            Integer c2 = termFrequency.get(term);
            if (c2 == null)
                c2 = 0;
            c2++;
            documentFrequency.put(term, c2);
        }
    }
    
    public void dumpTermFrequencies(String path) throws Exception
    {    
        // Open the Sesame repository
        String indexes = "spoc,ospc";
        NativeStore store = new NativeStore(new File(path), indexes);
        Repository repository = new SailRepository(store);
        repository.initialize();
        ElmoModule module = new ElmoModule();
        SesameManagerFactory factory = new SesameManagerFactory(module, repository);
        SesameManager manager = factory.createElmoManager(); 
        
        int conceptId = 0;
        for (Concept concept:  manager.findAll(Concept.class))
        {
            //QName qname = concept.getQName();
            //String uri = qname.getNamespaceURI() + qname.getLocalPart();
            
            // Pref label
            String prefLabel = concept.getSkosPrefLabel();
            countFrequencies(conceptId, prefLabel);

            // Alt labels
            Set<String> altLabels = concept.getSkosAltLabels();
            for (String altLabel: altLabels)
            {
                countFrequencies(conceptId, altLabel);
            }
            
            // Scope notes
            Set<Object> scopeNotes = concept.getSkosScopeNotes();
            for (Object scopeNote: scopeNotes)
            {
                String sn = (String)scopeNote;
                countFrequencies(conceptId, sn);
            }
            conceptId++;
        }
        repository.shutDown();
        store.shutDown();
        
        FileWriter fw = new FileWriter("TermFrequency.txt");
        Frequency termF = new Frequency();
        Frequency docF = new Frequency();
        
        for (String term: termFrequency.keySet())
        {
            Integer termFreq = termFrequency.get(term);
            termF.addValue(termFreq);
            Integer docFreq = documentFrequency.get(term);
            docF.addValue(docFreq);
            fw.write(term + "|" + termFreq + "|" + docFreq + "\n");
        }
        fw.flush();
        
        Collection<Integer> docVals = documentFrequency.values();
        Bag<Integer> uniqueDocVals = new TreeBag<Integer>();
        for (Integer val: docVals)
            uniqueDocVals.add(val);
            
        DecimalFormat df = new DecimalFormat("##.#####");

        for (Integer val: uniqueDocVals.uniqueSet())
        {
            double pct = docF.getPct(val);
            fw.write(val + "|" + df.format(pct) + "\n");
        }
        fw.close();
        
    }
    
    public Bag<String> getCandidates(String phrase) throws Exception
    {
        Bag<String> candidates = new HashBag<String>();
        

        IndexSearcher searcher = new IndexSearcher(dir);
        MultiFieldQueryParser p = new MultiFieldQueryParser(Version.LUCENE_34,
                new String[] { "prefLabel", "altLabel", "scopeNote"}, new MyAnalyzer());
        p.setDefaultOperator(Operator.AND);
        Query q =  p.parse(phrase);
        // 95% of words occur in 5 or fewer concepts if scope notes are not included
        // 90% of words occur in 10 or fewer concepts if scope notes are included
        TopDocs topDocs = searcher.search(q, 50);
        ScoreDoc[] hits = topDocs.scoreDocs;
        
        if (hits.length == 0 || hits.length > 10)
            return candidates;
        System.out.println(">> " + phrase);
        System.out.println(">> Results " + hits.length);
        int c = 0;
        float firstScore = 0;
        for (ScoreDoc hit: hits)
        {
            if (c == 0)
                firstScore = hit.score;
            
            
            float score = hit.score;
            if ((firstScore/score) > 2)
                break;
            
            Document doc = searcher.doc(hit.doc);
            Field cid = doc.getField("conceptId");
            candidates.add(cid.stringValue());
            Field prefLabel = doc.getField("prefLabel");
            System.out.println(prefLabel.stringValue() + "," + score);
            c++;
        }
        System.out.println("---------------\n");
        return candidates;
    }
    public void loadStopwords(String path) throws Exception
    {
        stopwords = FileUtils.readLines(new File(path), "UTF-8");
    }
    
}
